package cluster

import (
	"context"
	"crypto"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"gitee.com/iscas-system/cluster/pkg/clientaccess"
	"github.com/rancher/wrangler/pkg/generic"
	"github.com/rancher/wrangler/pkg/start"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"math"
	"math/big"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"io"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"path/filepath"

	"gitee.com/iscas-system/cluster/pkg/daemons/config"
	"gitee.com/iscas-system/cluster/pkg/version"
	"github.com/gorilla/mux"

	"github.com/rancher/wrangler/pkg/generated/controllers/core"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilsnet "k8s.io/utils/net"
)

func NewBacked(storage TLSStorage) TLSStorage {
	return &memory{storage: storage}
}

type memory struct {
	storage TLSStorage
	secret  *v1.Secret
}

func (m *memory) Get() (*v1.Secret, error) {
	if m.secret == nil && m.storage != nil {
		secret, err := m.storage.Get()
		if err != nil {
			return nil, err
		}
		m.secret = secret
	}

	return m.secret, nil
}

func (m *memory) Update(secret *v1.Secret) error {
	if m.secret == nil || m.secret.ResourceVersion == "" || m.secret.ResourceVersion != secret.ResourceVersion {
		if m.storage != nil {
			if err := m.storage.Update(secret); err != nil {
				return err
			}
		}

		logrus.Infof("Active TLS secret %s/%s (ver=%s) (count %d): %v", secret.Namespace, secret.Name, secret.ResourceVersion, len(secret.Annotations)-1, secret.Annotations)
		m.secret = secret
	}
	return nil
}

type tcpKeepAliveListener struct {
	*net.TCPListener
}

func NewTCPListener(ip string, port int) (net.Listener, error) {
	l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
	if err != nil {
		return nil, err
	}

	tcpListener, ok := l.(*net.TCPListener)
	if !ok {
		return nil, fmt.Errorf("wrong listener type: %v", reflect.TypeOf(tcpListener))
	}

	return tcpKeepAliveListener{
		TCPListener: tcpListener,
	}, nil
}

type TLSStorage interface {
	Get() (*v1.Secret, error)
	Update(secret *v1.Secret) error
}

type TLSFactory interface {
	Renew(secret *v1.Secret) (*v1.Secret, error)
	AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error)
	Merge(target *v1.Secret, additional *v1.Secret) (*v1.Secret, bool, error)
	Filter(cn ...string) []string
	Regenerate(secret *v1.Secret) (*v1.Secret, error)
}

type Config struct {
	CN                    string
	Organization          []string
	TLSConfig             *tls.Config
	SANs                  []string
	MaxSANs               int
	ExpirationDaysCheck   int
	CloseConnOnCertChange bool
	RegenerateCerts       func() bool
	FilterCN              func(...string) []string
}

type closeWrapper struct {
	net.Conn
	id    int
	l     *listener
	ready bool
}

type listener struct {
	sync.RWMutex
	net.Listener

	conns    map[int]*closeWrapper
	connID   int
	connLock sync.Mutex

	factory   TLSFactory
	storage   TLSStorage
	version   string
	tlsConfig *tls.Config
	cert      *tls.Certificate
	certReady chan struct{}
	sans      []string
	maxSANs   int
	init      sync.Once
}

func allowDefaultSANs(sans []string, next func(...string) []string) func(...string) []string {
	if next == nil {
		return nil
	} else if len(sans) == 0 {
		return next
	}

	sanMap := map[string]bool{}
	for _, san := range sans {
		sanMap[san] = true
	}

	return func(s ...string) []string {
		var (
			good    []string
			unknown []string
		)
		for _, s := range s {
			if sanMap[s] {
				good = append(good, s)
			} else {
				unknown = append(unknown, s)
			}
		}

		return append(good, next(unknown...)...)
	}
}

type nonNil struct {
	sync.Mutex
	storage TLSStorage
}

func (n *nonNil) Get() (*v1.Secret, error) {
	n.Lock()
	defer n.Unlock()

	s, err := n.storage.Get()
	if err != nil || s == nil {
		return &v1.Secret{}, err
	}
	return s, nil
}

func (n *nonNil) Update(secret *v1.Secret) error {
	n.Lock()
	defer n.Unlock()

	return n.storage.Update(secret)
}

func (l *listener) updateCert(cn ...string) error {
	cn = l.factory.Filter(cn...)
	if len(cn) == 0 {
		return nil
	}

	l.RLock()
	defer l.RUnlock()

	secret, err := l.storage.Get()
	if err != nil {
		return err
	}

	if IsStatic(secret) || !NeedsUpdate(l.maxSANs, secret, cn...) {
		return nil
	}

	l.RUnlock()
	l.Lock()
	defer l.RLock()
	defer l.Unlock()

	secret, updated, err := l.factory.AddCN(secret, append(l.sans, cn...)...)
	if err != nil {
		return err
	}

	if updated {
		if err := l.storage.Update(secret); err != nil {
			return err
		}
		// Clear version to force cert reload next time loadCert is called by TLSConfig's
		// GetCertificate hook to provide a certificate for a new connection. Note that this
		// means the old certificate stays in l.cert until a new connection is made.
		l.version = ""
	}

	return nil
}

func (c *closeWrapper) close() error {
	delete(c.l.conns, c.id)
	return c.Conn.Close()
}

func (l *listener) loadCert(currentConn net.Conn) (*tls.Certificate, error) {
	l.RLock()
	defer l.RUnlock()

	secret, err := l.storage.Get()
	if err != nil {
		return nil, err
	}
	if l.cert != nil && l.version == secret.ResourceVersion && secret.ResourceVersion != "" {
		return l.cert, nil
	}

	defer l.RLock()
	l.RUnlock()
	l.Lock()
	defer l.Unlock()

	secret, err = l.storage.Get()
	if err != nil {
		return nil, err
	}
	if !clientaccess.IsValidTLSSecret(secret) {
		return l.cert, nil
	}
	if l.cert != nil && l.version == secret.ResourceVersion && secret.ResourceVersion != "" {
		return l.cert, nil
	}

	cert, err := tls.X509KeyPair(secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey])
	if err != nil {
		return nil, err
	}

	// cert has changed, close closeWrapper wrapped connections if this isn't the first load
	if currentConn != nil && l.conns != nil && l.cert != nil {
		l.connLock.Lock()
		for _, conn := range l.conns {
			// Don't close a connection that's in the middle of completing a TLS handshake
			if !conn.ready {
				continue
			}
			_ = conn.close()
		}
		l.connLock.Unlock()
	}

	// we can only close the ready channel once when the cert is first assigned
	canClose := l.cert == nil
	l.cert = &cert
	if canClose {
		close(l.certReady)
	}
	l.version = secret.ResourceVersion
	return l.cert, nil
}

func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
	newConn := hello.Conn
	if hello.ServerName != "" {
		if err := l.updateCert(hello.ServerName); err != nil {
			logrus.Errorf("dynamiclistener %s: failed to update cert with TLS ServerName: %v", l.Addr(), err)
			return nil, err
		}
	}
	connCert, err := l.loadCert(newConn)
	if connCert != nil && err == nil && newConn != nil && l.conns != nil {
		// if we were successfully able to load a cert and are closing connections on cert changes, mark newConn ready
		// this will allow us to close the connection if a future connection forces the cert to re-load
		wrapper, ok := newConn.(*closeWrapper)
		if !ok {
			logrus.Debugf("will not mark non-close wrapper connection from %s to %s as ready", newConn.RemoteAddr(), newConn.LocalAddr())
			return connCert, err
		}
		l.connLock.Lock()
		l.conns[wrapper.id].ready = true
		l.connLock.Unlock()
	}
	return connCert, err
}

type SetFactory interface {
	SetFactory(tls TLSFactory)
}

func (l *listener) cacheHandler() http.Handler {
	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
		h, _, err := net.SplitHostPort(req.Host)
		if err != nil {
			h = req.Host
		}

		ip := net.ParseIP(h)
		if len(ip) > 0 {
			for _, v := range req.Header["User-Agent"] {
				if strings.Contains(strings.ToLower(v), "mozilla") {
					return
				}
			}

			if err := l.updateCert(h); err != nil {
				logrus.Errorf("dynamiclistener %s: failed to update cert with HTTP request Host header: %v", l.Addr(), err)
			}
		}
	})
}

// regenerateCerts regenerates the used certificates and
// updates the secret.
func (l *listener) regenerateCerts() error {
	l.Lock()
	defer l.Unlock()

	secret, err := l.storage.Get()
	if err != nil {
		return err
	}

	newSecret, err := l.factory.Regenerate(secret)
	if err != nil {
		return err
	}
	if err := l.storage.Update(newSecret); err != nil {
		return err
	}
	// clear version to force cert reload
	l.version = ""

	return nil
}

// getAnnotationKey return the key to use for a given CN. IPv4 addresses and short hostnames
// are safe to store as-is, but longer hostnames and IPv6 address must be truncated and/or undergo
// character replacement in order to be used as an annotation key. If the CN requires modification,
// a portion of the SHA256 sum of the original value is used as the suffix, to reduce the likelihood
// of collisions in modified values.
func getAnnotationKey(cn string) string {
	cn = cnPrefix + cn
	cnLen := len(cn)
	if cnLen < 64 && !strings.ContainsRune(cn, ':') {
		return cn
	}
	digest := sha256.Sum256([]byte(cn))
	cn = strings.ReplaceAll(cn, ":", "_")
	if cnLen > 56 {
		cnLen = 56
	}
	return cn[0:cnLen] + "-" + hex.EncodeToString(digest[0:])[0:6]
}

func (t *TLS) Verify(secret *v1.Secret) error {
	certsPem := secret.Data[v1.TLSCertKey]
	if len(certsPem) == 0 {
		return nil
	}

	certificates, err := clientaccess.ParseCertsPEM(certsPem)
	if err != nil || len(certificates) == 0 {
		return err
	}

	verifyOpts := x509.VerifyOptions{
		Roots: x509.NewCertPool(),
		KeyUsages: []x509.ExtKeyUsage{
			x509.ExtKeyUsageAny,
		},
	}
	for _, c := range t.CACert {
		verifyOpts.Roots.AddCert(c)
	}

	_, err = certificates[0].Verify(verifyOpts)
	return err
}

var (
	cnRegexp = regexp.MustCompile("^([A-Za-z0-9:][-A-Za-z0-9_.:]*)?[A-Za-z0-9:]$")
)

func populateCN(secret *v1.Secret, cn ...string) *v1.Secret {
	secret = secret.DeepCopy()
	if secret.Annotations == nil {
		secret.Annotations = map[string]string{}
	}
	for _, cn := range cn {
		if cnRegexp.MatchString(cn) {
			secret.Annotations[getAnnotationKey(cn)] = cn
		} else {
			logrus.Errorf("dropping invalid CN: %s", cn)
		}
	}
	return secret
}

func getPrivateKey(secret *v1.Secret) (crypto.Signer, error) {
	keyBytes := secret.Data[v1.TLSPrivateKeyKey]
	if len(keyBytes) == 0 {
		return NewPrivateKey()
	}

	privateKey, err := clientaccess.ParsePrivateKeyPEM(keyBytes)
	if signer, ok := privateKey.(crypto.Signer); ok && err == nil {
		return signer, nil
	}

	return NewPrivateKey()
}

// NewPrivateKey returnes a new ECDSA key
func NewPrivateKey() (crypto.Signer, error) {
	return ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
}

func collectCNs(secret *v1.Secret) (domains []string, ips []net.IP, err error) {
	var (
		cns = cns(secret)
	)

	sort.Strings(cns)

	for _, v := range cns {
		ip := net.ParseIP(v)
		if ip == nil {
			domains = append(domains, v)
		} else {
			ips = append(ips, ip)
		}
	}

	return
}

func (t *TLS) newCert(domains []string, ips []net.IP, privateKey crypto.Signer) (*x509.Certificate, error) {
	return NewSignedCert(privateKey, t.CACert[0], t.CAKey, t.CN, t.Organization, domains, ips)
}
func NewSignedCert(signer crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer, cn string, orgs []string,
	domains []string, ips []net.IP) (*x509.Certificate, error) {

	serialNumber, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
	if err != nil {
		return nil, err
	}

	expirationDays := defaultNewSignedCertExpirationDays
	envExpirationDays := os.Getenv("CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS")
	if envExpirationDays != "" {
		if envExpirationDaysInt, err := strconv.Atoi(envExpirationDays); err != nil {
			logrus.Infof("[NewSignedCert] expiration days from ENV (%s) could not be converted to int (falling back to default value: %d)", envExpirationDays, defaultNewSignedCertExpirationDays)
		} else {
			expirationDays = envExpirationDaysInt
		}
	}

	parent := x509.Certificate{
		DNSNames:     domains,
		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		IPAddresses:  ips,
		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		NotAfter:     time.Now().Add(time.Hour * 24 * time.Duration(expirationDays)).UTC(),
		NotBefore:    caCert.NotBefore,
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			CommonName:   cn,
			Organization: orgs,
		},
	}

	cert, err := x509.CreateCertificate(rand.Reader, &parent, caCert, signer.Public(), caKey)
	if err != nil {
		return nil, err
	}

	parsedCert, err := x509.ParseCertificate(cert)
	if err == nil {
		logrus.Infof("certificate %s signed by %s: notBefore=%s notAfter=%s",
			parsedCert.Subject, caCert.Subject, parsedCert.NotBefore, parsedCert.NotAfter)
	}
	return parsedCert, err
}

// MarshalChain returns given key and certificates as byte slices.
func MarshalChain(privateKey crypto.Signer, certs ...*x509.Certificate) (keyBytes, certChainBytes []byte, err error) {
	keyBytes, err = clientaccess.MarshalPrivateKeyToPEM(privateKey)
	if err != nil {
		return nil, nil, err
	}

	for _, cert := range certs {
		if cert != nil {
			certBlock := pem.Block{
				Type:  CertificateBlockType,
				Bytes: cert.Raw,
			}
			certChainBytes = append(certChainBytes, pem.EncodeToMemory(&certBlock)...)
		}
	}
	return keyBytes, certChainBytes, nil
}

func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
	secret = secret.DeepCopy()
	if secret == nil {
		secret = &v1.Secret{}
	}

	if err := t.Verify(secret); err != nil {
		logrus.Warnf("unable to verify existing certificate: %v - signing operation may change certificate issuer", err)
	}

	secret = populateCN(secret, cn...)

	privateKey, err := getPrivateKey(secret)
	if err != nil {
		return nil, false, err
	}

	domains, ips, err := collectCNs(secret)
	if err != nil {
		return nil, false, err
	}

	newCert, err := t.newCert(domains, ips, privateKey)
	if err != nil {
		return nil, false, err
	}

	keyBytes, certBytes, err := MarshalChain(privateKey, append([]*x509.Certificate{newCert}, t.CACert...)...)
	if err != nil {
		return nil, false, err
	}

	if secret.Data == nil {
		secret.Data = map[string][]byte{}
	}
	secret.Type = v1.SecretTypeTLS
	secret.Data[v1.TLSCertKey] = certBytes
	secret.Data[v1.TLSPrivateKeyKey] = keyBytes
	secret.Annotations[fingerprint] = fmt.Sprintf("SHA1=%X", sha1.Sum(newCert.Raw))

	return secret, true, nil
}

func cns(secret *v1.Secret) (cns []string) {
	if secret == nil {
		return nil
	}
	for k, v := range secret.Annotations {
		if strings.HasPrefix(k, cnPrefix) {
			cns = append(cns, v)
		}
	}
	return
}

type TLS struct {
	CACert              []*x509.Certificate
	CAKey               crypto.Signer
	CN                  string
	Organization        []string
	FilterCN            func(...string) []string
	ExpirationDaysCheck int
}

func (t *TLS) IsExpired(secret *v1.Secret) bool {
	certsPem := secret.Data[v1.TLSCertKey]
	if len(certsPem) == 0 {
		return false
	}

	certificates, err := clientaccess.ParseCertsPEM(certsPem)
	if err != nil || len(certificates) == 0 {
		return false
	}

	expirationDays := time.Duration(t.ExpirationDaysCheck) * time.Hour * 24
	return time.Now().Add(expirationDays).After(certificates[0].NotAfter)
}

// Merge combines the SAN lists from the target and additional Secrets, and
// returns a potentially modified Secret, along with a bool indicating if the
// returned Secret is not the same as the target Secret. Secrets with expired
// certificates will never be returned.
//
// If the merge would not add any CNs to the additional Secret, the additional
// Secret is returned, to allow for certificate rotation/regeneration.
//
// If the merge would not add any CNs to the target Secret, the target Secret is
// returned; no merging is necessary.
//
// If neither certificate is acceptable as-is, a new certificate containing
// the union of the two lists is generated, using the private key from the
// first Secret. The returned Secret will contain the updated cert.
func (t *TLS) Merge(target, additional *v1.Secret) (*v1.Secret, bool, error) {
	// static secrets can't be altered, don't bother trying
	if IsStatic(target) {
		return target, false, nil
	}

	mergedCNs := append(cns(target), cns(additional)...)

	// if the additional secret already has all the CNs, use it in preference to the
	// current one. This behavior is required to allow for renewal or regeneration.
	if !NeedsUpdate(0, additional, mergedCNs...) && !t.IsExpired(additional) {
		return additional, true, nil
	}

	// if the target secret already has all the CNs, continue using it. The additional
	// cert had only a subset of the current CNs, so nothing needs to be added.
	if !NeedsUpdate(0, target, mergedCNs...) && !t.IsExpired(target) {
		return target, false, nil
	}

	// neither cert currently has all the necessary CNs or is unexpired; generate a new one.
	return t.generateCert(target, mergedCNs...)
}

// Renew returns a copy of the given certificate that has been re-signed
// to extend the NotAfter date. It is an error to attempt to renew
// a static (user-provided) certificate.
func (t *TLS) Renew(secret *v1.Secret) (*v1.Secret, error) {
	if IsStatic(secret) {
		return secret, clientaccess.ErrStaticCert
	}
	cns := cns(secret)
	secret = secret.DeepCopy()
	secret.Annotations = map[string]string{}
	secret, _, err := t.generateCert(secret, cns...)
	return secret, err
}

// Filter ensures that the CNs are all valid accorting to both internal logic, and any filter callbacks.
// The returned list will contain only approved CN entries.
func (t *TLS) Filter(cn ...string) []string {
	if len(cn) == 0 || t.FilterCN == nil {
		return cn
	}
	return t.FilterCN(cn...)
}

// AddCN attempts to add a list of CN strings to a given Secret, returning the potentially-modified
// Secret along with a bool indicating whether or not it has been updated. The Secret will not be changed
// if it has an attribute indicating that it is static (aka user-provided), or if no new CNs were added.
func (t *TLS) AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
	cn = t.Filter(cn...)

	if IsStatic(secret) || !NeedsUpdate(0, secret, cn...) {
		return secret, false, nil
	}
	return t.generateCert(secret, cn...)
}

func (t *TLS) Regenerate(secret *v1.Secret) (*v1.Secret, error) {
	cns := cns(secret)
	secret, _, err := t.generateCert(nil, cns...)
	return secret, err
}

// NeedsUpdate returns true if any of the CNs are not currently present on the
// secret's Certificate, as recorded in the cnPrefix annotations. It will return
// false if all requested CNs are already present, or if maxSANs is non-zero and has
// been exceeded.
func NeedsUpdate(maxSANs int, secret *v1.Secret, cn ...string) bool {
	if secret == nil {
		return true
	}

	for _, cn := range cn {
		if secret.Annotations[getAnnotationKey(cn)] == "" {
			if maxSANs > 0 && len(cns(secret)) >= maxSANs {
				return false
			}
			return true
		}
	}

	return false
}

func NewListenerWithChain(l net.Listener, storage TLSStorage, caCert []*x509.Certificate, caKey crypto.Signer, config Config) (net.Listener, http.Handler, error) {
	if config.CN == "" {
		config.CN = "dynamic"
	}
	if len(config.Organization) == 0 {
		config.Organization = []string{"dynamic"}
	}
	if config.TLSConfig == nil {
		config.TLSConfig = &tls.Config{}
	}
	if config.ExpirationDaysCheck == 0 {
		config.ExpirationDaysCheck = 90
	}

	dynamicListener := &listener{
		factory: &TLS{
			CACert:              caCert,
			CAKey:               caKey,
			CN:                  config.CN,
			Organization:        config.Organization,
			FilterCN:            allowDefaultSANs(config.SANs, config.FilterCN),
			ExpirationDaysCheck: config.ExpirationDaysCheck,
		},
		Listener:  l,
		certReady: make(chan struct{}),
		storage:   &nonNil{storage: storage},
		sans:      config.SANs,
		maxSANs:   config.MaxSANs,
		tlsConfig: config.TLSConfig,
	}
	if dynamicListener.tlsConfig == nil {
		dynamicListener.tlsConfig = &tls.Config{}
	}
	dynamicListener.tlsConfig.GetCertificate = dynamicListener.getCertificate

	if config.CloseConnOnCertChange {
		if len(dynamicListener.tlsConfig.Certificates) == 0 {
			dynamicListener.tlsConfig.NextProtos = []string{"http/1.1"}
		}
		dynamicListener.conns = map[int]*closeWrapper{}
	}

	if setter, ok := storage.(SetFactory); ok {
		setter.SetFactory(dynamicListener.factory)
	}

	if config.RegenerateCerts != nil && config.RegenerateCerts() {
		if err := dynamicListener.regenerateCerts(); err != nil {
			return nil, nil, err
		}
	}

	tlsListener := tls.NewListener(dynamicListener.WrapExpiration(config.ExpirationDaysCheck), dynamicListener.tlsConfig)

	return tlsListener, dynamicListener.cacheHandler(), nil
}

func (l *listener) checkExpiration(days int) error {
	l.Lock()
	defer l.Unlock()

	if days == 0 {
		return nil
	}

	if l.cert == nil {
		return nil
	}

	secret, err := l.storage.Get()
	if err != nil {
		return err
	}

	keyPair, err := tls.X509KeyPair(secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey])
	if err != nil {
		return err
	}

	certParsed, err := x509.ParseCertificate(keyPair.Certificate[0])
	if err != nil {
		return err
	}

	if clientaccess.IsCertExpired(certParsed, days) {
		secret, err := l.factory.Renew(secret)
		if err != nil {
			return err
		}
		if err := l.storage.Update(secret); err != nil {
			return err
		}
		// clear version to force cert reload
		l.version = ""
	}

	return nil
}

type cancelClose struct {
	cancel func()
	net.Listener
}

var ErrStaticCert = errors.New("cannot renew static certificate")

func (l *listener) WrapExpiration(days int) net.Listener {
	ctx, cancel := context.WithCancel(context.Background())
	go func() {

		// wait for cert to be set, this will unblock when the channel is closed
		select {
		case <-ctx.Done():
			return
		case <-l.certReady:
		}

		for {
			wait := 6 * time.Hour
			if err := l.checkExpiration(days); err != nil {
				logrus.Errorf("dynamiclistener %s: failed to check and renew dynamic cert: %v", l.Addr(), err)
				// Don't go into short retry loop if we're using a static (user-provided) cert.
				// We will still check and print an error every six hours until the user updates the secret with
				// a cert that is not about to expire. Hopefully this will prompt them to take action.
				if err != ErrStaticCert {
					wait = 5 * time.Minute
				}
			}
			select {
			case <-ctx.Done():
				return
			case <-time.After(wait):
			}
		}
	}()

	return &cancelClose{
		cancel:   cancel,
		Listener: l,
	}
}

const (
	cnPrefix    = "listener.cattle.io/cn-"
	Static      = "listener.cattle.io/static"
	fingerprint = "listener.cattle.io/fingerprint"

	CertificateBlockType               = "CERTIFICATE"
	defaultNewSignedCertExpirationDays = 365
)

// IsStatic returns true if the Secret has an attribute indicating that it contains
// a static (aka user-provided) certificate, which should not be modified.
func IsStatic(secret *v1.Secret) bool {
	if secret != nil && secret.Annotations != nil {
		return secret.Annotations[Static] == "true"
	}
	return false
}

// newListener returns a new TCP listener and HTTP request handler using dynamiclistener.
// dynamiclistener will use the cluster's Server CA to sign the dynamically generate certificate,
// and will sync the certs into the Kubernetes datastore, with a local disk cache.
func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) {
	if c.managedDB != nil {
		resetDone, err := c.managedDB.IsReset()
		if err != nil {
			return nil, nil, err
		}
		if resetDone {
			// delete the dynamic listener TLS secret cache after restoring,
			// to ensure that dynamiclistener doesn't sync the old secret over the top
			// of whatever was just restored.
			os.Remove(filepath.Join(c.config.DataDir, "tls/dynamic-cert.json"))
		}
	}
	ip := c.config.BindAddress
	if utilsnet.IsIPv6String(ip) {
		ip = fmt.Sprintf("[%s]", ip)
	}
	tcp, err := NewTCPListener(ip, c.config.SupervisorPort)
	if err != nil {
		return nil, nil, err
	}
	certs, key, err := clientaccess.LoadCertsChain(c.config.Runtime.ServerCA, c.config.Runtime.ServerCAKey)
	if err != nil {
		return nil, nil, err
	}
	c.config.SANs = append(c.config.SANs, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc."+c.config.ClusterDomain)
	if c.config.SANSecurity {
		c.config.Runtime.ClusterControllerStarts["server-cn-filter"] = func(ctx context.Context) {
			registerAddressHandlers(ctx, c)
		}
	}
	storage := tlsStorage(ctx, c.config.DataDir, c.config.Runtime)
	return wrapHandler(NewListenerWithChain(tcp, storage, certs, key, Config{
		ExpirationDaysCheck: config.CertificateRenewDays,
		Organization:        []string{version.Program},
		SANs:                c.config.SANs,
		CN:                  version.Program,
		TLSConfig: &tls.Config{
			ClientAuth:   tls.RequestClientCert,
			MinVersion:   c.config.TLSMinVersion,
			CipherSuites: c.config.TLSCipherSuites,
			NextProtos:   []string{"h2", "http/1.1"},
		},
		FilterCN: c.filterCN,
		RegenerateCerts: func() bool {
			const regenerateDynamicListenerFile = "dynamic-cert-regenerate"
			dynamicListenerRegenFilePath := filepath.Join(c.config.DataDir, "tls", regenerateDynamicListenerFile)
			if _, err := os.Stat(dynamicListenerRegenFilePath); err == nil {
				os.Remove(dynamicListenerRegenFilePath)
				return true
			}
			return false
		},
	}))
}

func (c *Cluster) filterCN(cn ...string) []string {
	if c.cnFilterFunc != nil {
		return c.cnFilterFunc(cn...)
	}
	return cn
}

// initClusterAndHTTPS sets up the dynamic tls listener, request router,
// and cluster database. Once the database is up, it starts the supervisor http server.
func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error {
	// Set up dynamiclistener TLS listener and request handler
	listener, handler, err := c.newListener(ctx)
	if err != nil {
		return err
	}

	// Get the base request handler
	handler, err = c.getHandler(handler)
	if err != nil {
		return err
	}

	// Register database request handlers and controller callbacks
	handler, err = c.registerDBHandlers(handler)
	if err != nil {
		return err
	}

	if c.config.EnablePProf {
		mux := mux.NewRouter().SkipClean(true)
		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
		mux.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index)
		mux.NotFoundHandler = handler
		handler = mux
	}

	// Create a HTTP server with the registered request handlers, using logrus for logging
	server := http.Server{
		Handler: handler,
	}

	if logrus.IsLevelEnabled(logrus.DebugLevel) {
		server.ErrorLog = log.New(logrus.StandardLogger().Writer(), "Cluster-Http-Server ", log.LstdFlags)
	} else {
		server.ErrorLog = log.New(io.Discard, "Cluster-Http-Server", 0)
	}

	// Start the supervisor http server on the tls listener
	go func() {
		err := server.Serve(listener)
		if err != nil && !errors.Is(err, http.ErrServerClosed) {
			logrus.Fatalf("server stopped: %v", err)
		}
	}()

	// Shutdown the http server when the context is closed
	go func() {
		<-ctx.Done()
		server.Shutdown(context.Background())
	}()

	return nil
}

func NewStoarge(file string) TLSStorage {
	return &storage{
		file: file,
	}
}

type storage struct {
	file string
}

func (s *storage) Get() (*v1.Secret, error) {
	f, err := os.Open(s.file)
	if os.IsNotExist(err) {
		return nil, nil
	} else if err != nil {
		return nil, err
	}
	defer f.Close()

	secret := v1.Secret{}
	return &secret, json.NewDecoder(f).Decode(&secret)
}

func (s *storage) Update(secret *v1.Secret) error {
	f, err := os.Create(s.file)
	if err != nil {
		return err
	}
	defer f.Close()

	return json.NewEncoder(f).Encode(secret)
}

type CoreGetter func() *core.Factory

func NewKubernetes(ctx context.Context, core CoreGetter, namespace, name string, backing TLSStorage) TLSStorage {
	storage := &kube{
		name:      name,
		namespace: namespace,
		storage:   backing,
		ctx:       ctx,
		initSync:  &sync.Once{},
	}

	// lazy init
	go func() {
		wait.PollImmediateUntilWithContext(ctx, time.Second, func(cxt context.Context) (bool, error) {
			if coreFactory := core(); coreFactory != nil {
				storage.init(coreFactory.Core().V1().Secret())
				return true, start.All(ctx, 5, coreFactory)
			}
			return false, nil
		})
	}()

	return storage
}

func (s *kube) Get() (*v1.Secret, error) {
	s.RLock()
	defer s.RUnlock()

	return s.storage.Get()
}

var knownReasons = map[metav1.StatusReason]struct{}{
	// metav1.StatusReasonUnknown : {}
	metav1.StatusReasonUnauthorized:          {},
	metav1.StatusReasonForbidden:             {},
	metav1.StatusReasonNotFound:              {},
	metav1.StatusReasonAlreadyExists:         {},
	metav1.StatusReasonConflict:              {},
	metav1.StatusReasonGone:                  {},
	metav1.StatusReasonInvalid:               {},
	metav1.StatusReasonServerTimeout:         {},
	metav1.StatusReasonTimeout:               {},
	metav1.StatusReasonTooManyRequests:       {},
	metav1.StatusReasonBadRequest:            {},
	metav1.StatusReasonMethodNotAllowed:      {},
	metav1.StatusReasonNotAcceptable:         {},
	metav1.StatusReasonRequestEntityTooLarge: {},
	metav1.StatusReasonUnsupportedMediaType:  {},
	metav1.StatusReasonInternalError:         {},
	metav1.StatusReasonExpired:               {},
	metav1.StatusReasonServiceUnavailable:    {},
}

// IsNotFound returns true if the specified error was created by NewNotFound.
// It supports wrapped errors and returns false when the error is nil.
func IsNotFound(err error) bool {
	reason, code := reasonAndCodeForError(err)
	if reason == metav1.StatusReasonNotFound {
		return true
	}
	if _, ok := knownReasons[reason]; !ok && code == http.StatusNotFound {
		return true
	}
	return false
}

func (s *kube) init(secrets SecretController) {
	s.Lock()
	defer s.Unlock()

	secrets.OnChange(s.ctx, "tls-storage", func(key string, secret *v1.Secret) (*v1.Secret, error) {
		if secret == nil {
			return nil, nil
		}
		if secret.Namespace == s.namespace && secret.Name == s.name {
			if err := s.Update(secret); err != nil {
				return nil, err
			}
		}

		return secret, nil
	})
	s.secrets = secrets

	// Asynchronously sync the backing storage to the Kubernetes secret, as doing so inline may
	// block the listener from accepting new connections if the apiserver becomes unavailable
	// after the Secrets controller has been initialized. We're not passing around any contexts
	// here, nor does the controller accept any, so there's no good way to soft-fail with a
	// reasonable timeout.
	go s.syncStorage()
}

// IsAlreadyExists determines if the err is an error which indicates that a specified resource already exists.
// It supports wrapped errors and returns false when the error is nil.
func IsAlreadyExists(err error) bool {
	return ReasonForError(err) == metav1.StatusReasonAlreadyExists
}

// APIStatus is exposed by errors that can be converted to an api.Status object
// for finer grained details.
type APIStatus interface {
	Status() metav1.Status
}

// ReasonForError returns the HTTP status for a particular error.
// It supports wrapped errors and returns StatusReasonUnknown when
// the error is nil or doesn't have a status.
func ReasonForError(err error) metav1.StatusReason {
	if status, ok := err.(APIStatus); ok || errors.As(err, &status) {
		return status.Status().Reason
	}
	return metav1.StatusReasonUnknown
}

func IsValidTLSSecret(secret *v1.Secret) bool {
	if secret == nil {
		return false
	}
	if _, ok := secret.Data[v1.TLSCertKey]; !ok {
		return false
	}
	if _, ok := secret.Data[v1.TLSPrivateKeyKey]; !ok {
		return false
	}
	return true
}

func (s *kube) syncStorage() {
	var updateStorage bool
	secret, err := s.Get()
	if err == nil && IsValidTLSSecret(secret) {
		// local storage had a cached secret, ensure that it exists in Kubernetes
		_, err := s.secrets.Create(&v1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:        s.name,
				Namespace:   s.namespace,
				Annotations: secret.Annotations,
			},
			Type: v1.SecretTypeTLS,
			Data: secret.Data,
		})
		if err != nil && !IsAlreadyExists(err) {
			logrus.Warnf("Failed to create Kubernetes secret: %v", err)
		}
	} else {
		// local storage was empty, try to populate it
		secret, err = s.secrets.Get(s.namespace, s.name, metav1.GetOptions{})
		if err != nil {
			if !IsNotFound(err) {
				logrus.Warnf("Failed to init Kubernetes secret: %v", err)
			}
		} else {
			updateStorage = true
		}
	}

	s.Lock()
	defer s.Unlock()
	s.initialized = true
	if updateStorage {
		if err := s.storage.Update(secret); err != nil {
			logrus.Warnf("Failed to init backing storage secret: %v", err)
		}
	}
}

func (s *kube) Update(secret *v1.Secret) error {
	// Asynchronously update the Kubernetes secret, as doing so inline may block the listener from
	// accepting new connections if the apiserver becomes unavailable after the Secrets controller
	// has been initialized. We're not passing around any contexts here, nor does the controller
	// accept any, so there's no good way to soft-fail with a reasonable timeout.
	go func() {
		if err := s.update(secret); err != nil {
			logrus.Errorf("Failed to save TLS secret for %s/%s: %v", secret.Namespace, secret.Name, err)
		}
	}()
	return nil
}

// IsConflict determines if the err is an error which indicates the provided update conflicts.
// It supports wrapped errors and returns false when the error is nil.
func IsConflict(err error) bool {
	reason, code := reasonAndCodeForError(err)
	if reason == metav1.StatusReasonConflict {
		return true
	}
	if _, ok := knownReasons[reason]; !ok && code == http.StatusConflict {
		return true
	}
	return false
}

func reasonAndCodeForError(err error) (metav1.StatusReason, int32) {
	if status, ok := err.(APIStatus); ok || errors.As(err, &status) {
		return status.Status().Reason, status.Status().Code
	}
	return metav1.StatusReasonUnknown, 0
}

func isConflictOrAlreadyExists(err error) bool {
	return IsConflict(err) || IsAlreadyExists(err)
}

func (s *kube) initComplete() bool {
	s.RLock()
	defer s.RUnlock()
	return s.initialized
}

func (s *kube) targetSecret() (*v1.Secret, error) {
	s.RLock()
	defer s.RUnlock()

	existingSecret, err := s.secrets.Get(s.namespace, s.name, metav1.GetOptions{})
	if IsNotFound(err) {
		return &v1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      s.name,
				Namespace: s.namespace,
			},
			Type: v1.SecretTypeTLS,
		}, nil
	}
	return existingSecret, err
}

func (s *kube) saveInK8s(secret *v1.Secret) (*v1.Secret, error) {
	if !s.initComplete() {
		// Start a goroutine to attempt to save the secret later, once init is complete.
		// If this was already handled by initComplete, it should be a no-op, or at worst get
		// merged with the Kubernetes secret.
		go s.initSync.Do(func() {
			if err := wait.Poll(100*time.Millisecond, 15*time.Minute, func() (bool, error) {
				if !s.initComplete() {
					return false, nil
				}
				_, err := s.saveInK8s(secret)
				return true, err
			}); err != nil {
				logrus.Errorf("Failed to save TLS secret after controller init: %v", err)
			}
		})
		return secret, nil
	}

	targetSecret, err := s.targetSecret()
	if err != nil {
		return nil, err
	}

	// if we don't have a TLS factory we can't create certs, so don't bother trying to merge anything,
	// in favor of just blindly replacing the fields on the Kubernetes secret.
	if s.tls != nil {
		// merge new secret with secret from backing storage, if one exists
		if existing, err := s.Get(); err == nil && IsValidTLSSecret(existing) {
			if newSecret, updated, err := s.tls.Merge(existing, secret); err == nil && updated {
				secret = newSecret
			}
		}

		// merge new secret with existing secret from Kubernetes, if one exists
		if IsValidTLSSecret(targetSecret) {
			if newSecret, updated, err := s.tls.Merge(targetSecret, secret); err != nil {
				return nil, err
			} else if !updated {
				return newSecret, nil
			} else {
				secret = newSecret
			}
		}
	}

	// ensure that the merged secret actually contains data before overwriting the existing fields
	if !IsValidTLSSecret(secret) {
		logrus.Warnf("Skipping save of TLS secret for %s/%s due to missing certificate data", secret.Namespace, secret.Name)
		return targetSecret, nil
	}

	targetSecret.Annotations = secret.Annotations
	targetSecret.Type = v1.SecretTypeTLS
	targetSecret.Data = secret.Data

	if targetSecret.UID == "" {
		logrus.Infof("Creating new TLS secret for %s/%s (count: %d): %v", targetSecret.Namespace, targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
		return s.secrets.Create(targetSecret)
	}
	logrus.Infof("Updating TLS secret for %s/%s (count: %d): %v", targetSecret.Namespace, targetSecret.Name, len(targetSecret.Annotations)-1, targetSecret.Annotations)
	return s.secrets.Update(targetSecret)
}
func (s *kube) update(secret *v1.Secret) (err error) {
	var newSecret *v1.Secret
	err = retry.OnError(retry.DefaultRetry, isConflictOrAlreadyExists, func() error {
		newSecret, err = s.saveInK8s(secret)
		return err
	})

	if err != nil {
		return err
	}

	// Only hold the lock while updating underlying storage
	s.Lock()
	defer s.Unlock()
	return s.storage.Update(newSecret)
}

// SecretController interface for managing Secret resources.
type SecretController interface {
	generic.ControllerInterface[*v1.Secret, *v1.SecretList]
}

type kube struct {
	sync.RWMutex

	namespace, name string
	storage         TLSStorage
	secrets         SecretController
	ctx             context.Context
	tls             TLSFactory
	initialized     bool
	initSync        *sync.Once
}

// tlsStorage creates an in-memory cache for dynamiclistener's certificate, backed by a file on disk
// and the Kubernetes datastore.
func tlsStorage(ctx context.Context, dataDir string, runtime *config.ControlRuntime) TLSStorage {
	fileStorage := NewStoarge(filepath.Join(dataDir, "tls/dynamic-cert.json"))
	cache := NewBacked(fileStorage)
	return NewKubernetes(ctx, func() *core.Factory {
		return runtime.Core
	}, metav1.NamespaceSystem, version.Program+"-serving", cache)
}

// wrapHandler wraps the dynamiclistener request handler, adding a User-Agent value to
// CONNECT requests that will prevent DynamicListener from adding the request's Host
// header to the SAN list.  CONNECT requests set the Host header to the target of the
// proxy connection, so it is not correct to add this value to the certificate.  It would
// be nice if we could do this with with the FilterCN callback, but unfortunately that
// callback does not offer access to the request that triggered the change.
func wrapHandler(listener net.Listener, handler http.Handler, err error) (net.Listener, http.Handler, error) {
	return listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodConnect {
			r.Header.Add("User-Agent", "mozilla")
		}
		handler.ServeHTTP(w, r)
	}), err
}
